package cn.itcast.hdfs;

import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsApi {
    /**
     * 第一种方式
     */
    @Test
    public void getFileSystem1() {
        // 获取Configuration对象
        Configuration entries = new Configuration();
        //
        entries.set("fs.defaultFS", "hdfs://node01:8020");

        try {
            FileSystem fileSystem = FileSystem.get(entries);
            System.out.println(fileSystem.toString());
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 第二种方式
     */
    @Test
    public void getFileSystem2() throws URISyntaxException, IOException {
//        Configuration configuration = new Configuration();
//        configuration.set("fs.defaultFS","hdfs://192.168.52.250:8020");
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        System.out.println(fileSystem.toString());
    }

    /**
     * 第三种方式
     */
    @Test
    public void getFileSystem3() throws URISyntaxException, IOException {
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node01:8020"), configuration);
        System.out.println(fileSystem.toString());
    }

    /**
     * 第四种方式
     */
    @Test
    public void getFileSystem4() throws Exception {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://node01:8020");
        FileSystem fileSystem = FileSystem.newInstance(configuration);
        System.out.println(fileSystem.toString());
    }

    /**
     * 获取目录下的所有的文件信息
     */
    @Test
    public void listFile() throws Exception {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        RemoteIterator<LocatedFileStatus> iterator =
                fileSystem.listFiles(new Path("/"), true);
        while (iterator.hasNext()) {
            // 获取文件的详细信息
            LocatedFileStatus next = iterator.next();
            // 获取文件的存储路径
            System.out.println(next.getPath() + " " + next.getPath().getName());
            // 获取文件的block数
            BlockLocation[] blockLocations = next.getBlockLocations();
            System.out.println(blockLocations.length);
            long blockSize = next.getBlockSize();
            // 获取每一个文件的block的存储位置
            for (BlockLocation blockLocation : blockLocations) {
                String[] hosts = blockLocation.getHosts();
                System.out.println(hosts);
                for (String host : hosts) {
                    System.out.println(host + "-");
                }
            }
        }
    }

    /**
     * 创建文件  文件夹
     */
    @Test
    public void makdirs() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        // 创建文件夹
        fileSystem.mkdirs(new Path("/test/new"));
        // 创建文件
        fileSystem.create(new Path("/as.txt"));
    }

    /**
     * 文件的下载1
     */
    @Test
    public void donwloadFile() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        // 获取Hdfs文件
        FSDataInputStream inputStream = fileSystem.open(new Path("/app/知识点.txt"));
        // 获取本地文件
        FileOutputStream outputStream = new FileOutputStream(new File("C:/Users/zhong/Desktop/app/知识点.txt"));
        IOUtils.copy(inputStream, outputStream);
        IOUtils.closeQuietly(inputStream);
        IOUtils.closeQuietly(outputStream);
        fileSystem.close();


    }
    /**
     * 文件的下载2
     */
    @Test
    public void donwloadFile2() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        // 获取Hdfs文件
        fileSystem.copyToLocalFile(new Path("/app/知识点.txt"), new Path("C:/Users/zhong/Desktop/app1/知识点.txt"));
        fileSystem.close();


    }
    /**
     * 文件的下载2
     */
    @Test
    public void uploadFile2() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration());
        // 获取Hdfs文件
        fileSystem.copyFromLocalFile(new Path("C:/Users/zhong/Desktop/app/知识点.txt"),
                new Path("/app1/知识点.txt"));
        fileSystem.close();


    }
    /**
     * 合并文件
     */
    @Test
    public void mergeFile() throws URISyntaxException, IOException, InterruptedException {
        // 获取文件系统
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"), new Configuration(), "root");
        // 创建一个大文件
        FSDataOutputStream outputStream = fileSystem.create(new Path("/bigfile.txt"));
        // 获取本地文件
        LocalFileSystem local = FileSystem.getLocal(new Configuration());
        // 获取本地文件夹下所有文件
        FileStatus[] fileStatuses = local.listStatus(new Path("C:\\Users\\zhong\\Desktop\\test"));
        // 遍历每一个文件copy到大文件中
        for (FileStatus status : fileStatuses) {
            FSDataInputStream inputStream = local.open(status.getPath());
            IOUtils.copy(inputStream, outputStream);
            IOUtils.closeQuietly(inputStream);
        }
        outputStream.close();
        fileSystem.close();
        local.close();
    }
}
